'use strict';
const Axios = require('axios');
const LZString = require('../lib/LZStringJQS.js');
const util = require('../lib/util.js');
const querystring= require('querystring');
let cookies = [];
var latestDate = ''

async function getParam(ctx, count = 3) {
  if (count < 0) {
    return null;
  }
  var ua = util.getUa()
  var respose = await Axios({
    method: 'get',
    url: 'https://www3.wipo.int/branddb/en/',
    proxy: false,
    timeout: 120000,
    headers: {
      'user-agent': ua,
    },
  });
  cookies = [];
  for (let cookieStr of respose.headers['set-cookie']) {
    var cookie = {};
    cookieStr.split(';').forEach(Cookie => {
      var parts = Cookie.split('=');
      cookie[parts[0].trim()] = (parts[1] || '').trim();
    });
    cookies.push(cookie);
  }
  var resposeData = respose.data || null;
  try {
    // var len = resposeData.length;
    // var trafficStatistics = await ctx.app.redis.get('traffic-statistics') || 0;
    // await ctx.app.redis.set('traffic-statistics', len + Number(trafficStatistics));
    var qk = await util.findQkByDoc(resposeData);
    var fo = await util.findFoByDoc(resposeData);
    if (qk && fo) {
      ctx.app.logger.info(`qk: ${qk}, fo: ${fo}`);
      return {qk: qk, fo: fo};
    }
    return await getParam(ctx, --count);
  } catch (error) {
    ctx.app.logger.error('getParam error %s', error.message, error.stack);
    return await getParam(ctx, --count);
  }
}

async function docList(t, ctx, queryBase, tunnelProxy, nums, te, upStart, vals) {
  var respData = {}
  var setCookie = cookies;
  var data = `qz=${queryBase}`;
  var ua = util.getUa()
  var respose = await Axios({
    method: 'post',
    url: 'https://www3.wipo.int/branddb/jsp/select.jsp',
    data,
    timeout: 30000, 
    proxy: false,
    httpsAgent: tunnelProxy,
    headers: {
      'Host': 'www3.wipo.int',
      'User-Agent': ua,
      'Accept': 'application/json, text/javascript, */*; q=0.01',
      'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
      'Accept-Encoding': 'gzip, deflate, br',
      'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
      'Referer': 'https://www3.wipo.int/branddb/en/',
      'X-Requested-With': 'XMLHttpRequest',
      'Cookie': setCookie.join('; ')
    },
  }).catch(error => {
    throw new Error(error.message);
  });
  try {
    var resposeData = respose.data || {};
    var resp = resposeData.response || {};
    if (resposeData.qi && resp.docs.length > 0) {
      resp.qi = resposeData.qi;
      resp.qk = resposeData.qk;
      resp.lastUpdated = resposeData.lastUpdated;
      let docs = resp.docs
      let source = docs[0].SOURCE
      // var numFound = resp.numFound
      var docslist = []
      // if(Number(numFound) > Number(startFrom) && i == 0) {nums = Math.ceil((Number(numFound)-Number(startFrom))/100)}
      if(!t) {
        if(docs.length >= 100) {
          nums += 1
        }
        if(upStart == 0) {
          var teInfor = {}
          teInfor.startFrom = vals.startFrom
          teInfor.totalNums = vals.totalNums
          teInfor.status = vals.status
          teInfor.latestDate = Date.parse(docs[0].AD) - 28800000
          await ctx.app.redis.hset('tortious_config', source, querystring.stringify(teInfor));
          latestDate = teInfor.latestDate
        }
      }else{
        docs.forEach(function(item){ 
          if(Date.parse(item.AD) - 28800000 >= vals.latestDate && vals.latestDate > ctx.app.config.datas_start_time) {
            docslist.push(item)
          }
        });
        if(docslist.length < 1) {
          ctx.app.logger.info(`${t}//${source}//startFrom(${upStart})//length(${docslist.length}): (new)noData update!`);
          respData.nums = nums
          return respData
        }
        if(docslist.length >= 100) {
          nums += 1
        }
        resp.docs = docslist
      }
      respData.resp = resp
    }else{
      ctx.app.logger.error(`${t}//${te}//startFrom(${upStart})//resp(${JSON.stringify(resp)})`);
    }
  } catch (error) {
    ctx.app.logger.error(`${t}//${te}//startFrom(${upStart})//resposeData(${JSON.stringify(resposeData)}): get resp fail!: `, error.message);
  }
  respData.nums = nums
  return respData
}

async function main(t, ctx, app) {
  var tunnelProxy;
  var maxConn = app.config.mysql_max_connections;
  var params = await getParam(ctx)
  if(!params) {
    ctx.app.logger.error('getParam fail!');
    return
  };
  var qk = `0-${params.qk}`
  var fo = params.fo
  var queryCofig = app.config.queryCofig;
  queryCofig.qi = qk;
  queryCofig._ = fo;
  var statusInfo = {
    'status': 0,
    'latestDate': Date.now()
  };
  await app.redis.hset('tortious_run', 'status', querystring.stringify(statusInfo));
  var keys = await app.redis.hkeys('tortious_config');
  var totalNums = 0;
  for(let index = 0; index < keys.length; index++) {
    var te = keys[index];
    var vals = await app.redis.hget('tortious_config', te);
    vals = querystring.parse(vals);
    var startFrom = vals.startFrom;
    var status = vals.status;
    if(t && vals.startFrom == '0') {
      app.logger.info(`${t}//${te} finish!: all-tortious hasn't been run!`);
      continue
    };
    queryCofig.p.search.fq[0].te = status;
    queryCofig.p.search.fq[1].te = te;
    if(te == 'MAD') {
      queryCofig.p.search.fq[1].dt = 'WOTM'
    };
    var nums = 1;
    var errNums = 0;
    var reqListArr = [];
    // get response
    for (let i = 0; i<nums; i++){
      try {
        if(!t) {
          if(reqListArr.length >= maxConn) break;
          var upStart = Number(startFrom) + i*100;
        }else {
          var upStart = 0 + i*100;
        }
        queryCofig.p.start = upStart;
        var queryBase = LZString.getCurrentStateString(queryCofig);
        if(!queryBase) {
          app.logger.error('get queryBase fail!: %s', error.message);
          return
        }
        // if(!tunnelProxy || i%2 == 0) {
        //   try {
        //     tunnelProxy = await util.initTunnelProxy();
        //   } catch (error) {
        //     if(t) {
        //       reqListArr = [];
        //     }
        //     app.logger.error('get proxy fail!: %s', error.message);
        //     break;
        //   }
        // }
        try {
          tunnelProxy = await util.initTunnelProxy();
        } catch (error) {
          if(t) {
            reqListArr = [];
          }
          app.logger.error('get proxy fail!: %s', error.message);
          break;
        }
        if(tunnelProxy) {
          var respData = await docList(t, ctx, queryBase, tunnelProxy, nums, te, upStart, vals);
          nums = respData.nums;
          if(respData.resp) {
            reqListArr.push(respData.resp);
            // app.logger.info(`${t}//${te}//startFrom(${upStart})//length(${respData.resp.docs.length}): get resp ok!`);
          }
        }else {
          throw new Error('tunnelProxy is null');
        }
      } catch (error) {
      	// app.logger.error('get response fail!: %s', error.message);
        if(errNums < 500) {
          errNums++;
          i--;
          try {
            tunnelProxy = await util.initTunnelProxy();
          } catch (error) {
            if(t) {
              reqListArr = [];
            }
            app.logger.error('get proxy fail!: %s', error.message);
            break;
          }
        }
      }
    }
    // save response
    var repsarr = await Promise.all(reqListArr);
    var pList = [];
    var getDocsLength = 0
    var repsStart = ''
    for (let index = 0; index < repsarr.length; index++) {
      try {
        if (Array.isArray(repsarr[index].docs) && repsarr[index].docs.length > 0) {
          repsStart = repsarr[index].start
          var data = {}
          data.ctx = ctx
          data.vals = vals
          data.t = t
          data.latestDate = latestDate          
          pList.push(ctx.service.tortious.changeTortious(repsarr[index], data))
          // var sour = repsarr[index].docs[0].SOURCE
          // var nfd = repsarr[index].numFound
          // var ups = Number(repsStart)
          // ctx.app.logger.info(`${t}//${sour}//total(${nfd})//startFrom(${ups})//length(${repsarr[index].docs.length}): save ok!`);
          getDocsLength += repsarr[index].docs.length;
        }
      } catch (error) {
        app.logger.error(`${t}//${te}//startFrom(${repsStart}) save resp fail!: ${error}`);
      }
    }
    await Promise.all(pList).then(function() {
      app.logger.info(`${t}//${te}//startFrom(${startFrom}) saveOrUpdate datas totalNums: ${getDocsLength}!`);
      app.logger.info(`${t} source :${te} finish!`);
      totalNums += getDocsLength;
    })
  }
  await app.redis.hset('tortious_record', util.nowstr(), `${t} saveOrUpdate datas totalNums: ${totalNums}`);
  app.logger.info(`${t} finish!!!`);
}

module.exports = app => {
  return {
    schedule: {
      interval: app.config.all_schedule.interval,
      type: 'worker',
      immediate: app.config.all_schedule.immediate,
      disable: app.config.all_schedule.disable,
    },
    async task(ctx) {
      // startStatus: 0--closed, 1--run all and new, 2--run new
      var statusVuls = await ctx.app.redis.hget('tortious_run', 'status');
      var startStatus = querystring.parse(statusVuls).status;
      var startDate = querystring.parse(statusVuls).latestDate;
      if(startStatus == '0') {
        if(Date.now() - startDate <= app.config.run_max_time) {
          app.logger.info('spider is runing or closed!')
          return
        }
        else {
          startStatus == '1'
        }
      };
      app.logger.info('tortious_run: ', startStatus)
      try {
        if(startStatus == '1') {
          // main: 0--run all, 1--run new
          var p = [];
          var result = await main(0, ctx, app)
          p.push(result)
          await Promise.all(p).then()  
        };
        await main(1, ctx, app)
      } catch (error) {
        app.logger.error(`task err; ${error}`)
        await app.redis.hset('tortious_record', util.nowstr(), `task error: ${error}`);
      } finally {
        var statusInfo = {
          'status': startStatus,
          'latestDate': Date.now()
        };
        await app.redis.hset('tortious_run', 'status', querystring.stringify(statusInfo));
      }
    }
  }
}
